{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0804 19:50:35.437467 11296 deprecation_wrapper.py:119] From D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import pickle\n",
    "import bert\n",
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import shuffle\n",
    "from pyhanlp import *\n",
    "from bert import run_classifier\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import argparse\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretty_print(result):\n",
    "    df = pd.DataFrame([result]).T\n",
    "    df.columns = [\"values\"]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_tokenizer_from_hub_module(bert_model_hub):\n",
    "    \"\"\"Get the vocab file and casing info from the Hub module.\"\"\"\n",
    "    with tf.Graph().as_default():\n",
    "        bert_module = hub.Module(bert_model_hub)\n",
    "        tokenization_info = bert_module(signature=\"tokenization_info\", as_dict=True)\n",
    "        with tf.Session() as sess:\n",
    "            vocab_file, do_lower_case = sess.run([tokenization_info[\"vocab_file\"],\n",
    "                                            tokenization_info[\"do_lower_case\"]])\n",
    "      \n",
    "    return bert.tokenization.FullTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_features(dataset, label_list, MAX_SEQ_LENGTH, tokenizer, DATA_COLUMN, LABEL_COLUMN):\n",
    "    input_example = dataset.apply(lambda x: bert.run_classifier.InputExample(guid=None, \n",
    "                                                                   text_a = x[DATA_COLUMN], \n",
    "                                                                   text_b = None, \n",
    "                                                                   label = x[LABEL_COLUMN]), axis = 1)\n",
    "    features = bert.run_classifier.convert_examples_to_features(input_example, label_list, MAX_SEQ_LENGTH, tokenizer)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(bert_model_hub, is_predicting, input_ids, input_mask, segment_ids, labels, num_labels):\n",
    "    \"\"\"Creates a classification model.\"\"\"\n",
    "\n",
    "    bert_module = hub.Module(bert_model_hub, trainable=True)\n",
    "    bert_inputs = dict(input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids)\n",
    "    bert_outputs = bert_module(inputs=bert_inputs, signature=\"tokens\", as_dict=True)\n",
    "\n",
    "    # Use \"pooled_output\" for classification tasks on an entire sentence.\n",
    "    # Use \"sequence_outputs\" for token-level output.\n",
    "    output_layer = bert_outputs[\"pooled_output\"]\n",
    "\n",
    "    hidden_size = output_layer.shape[-1].value\n",
    "\n",
    "    # Create our own layer to tune for politeness data.\n",
    "    output_weights = tf.get_variable(\n",
    "        \"output_weights\", [num_labels, hidden_size],\n",
    "        initializer=tf.truncated_normal_initializer(stddev=0.02))\n",
    "\n",
    "    output_bias = tf.get_variable(\n",
    "        \"output_bias\", [num_labels], initializer=tf.zeros_initializer())\n",
    "\n",
    "    with tf.variable_scope(\"loss\"):\n",
    "\n",
    "        # Dropout helps prevent overfitting\n",
    "        output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)\n",
    "\n",
    "        logits = tf.matmul(output_layer, output_weights, transpose_b=True)\n",
    "        logits = tf.nn.bias_add(logits, output_bias)\n",
    "        log_probs = tf.nn.log_softmax(logits, axis=-1)\n",
    "\n",
    "        # Convert labels into one-hot encoding\n",
    "        one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)\n",
    "\n",
    "        predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))\n",
    "        # If we're predicting, we want predicted labels and the probabiltiies.\n",
    "        if is_predicting:\n",
    "            return (predicted_labels, log_probs)\n",
    "\n",
    "        # If we're train/eval, compute loss between predicted and actual label\n",
    "        per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)\n",
    "        loss = tf.reduce_mean(per_example_loss)\n",
    "        return (loss, predicted_labels, log_probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn_builder(bert_model_hub, num_labels, learning_rate, num_train_steps, num_warmup_steps):\n",
    "    \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n",
    "    def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument\n",
    "        \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n",
    "\n",
    "        input_ids = features[\"input_ids\"]\n",
    "        input_mask = features[\"input_mask\"]\n",
    "        segment_ids = features[\"segment_ids\"]\n",
    "        label_ids = features[\"label_ids\"]\n",
    "\n",
    "        is_predicting = (mode == tf.estimator.ModeKeys.PREDICT)\n",
    "    \n",
    "        # TRAIN and EVAL\n",
    "        if not is_predicting:\n",
    "\n",
    "            (loss, predicted_labels, log_probs) = create_model(\n",
    "            bert_model_hub, is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n",
    "\n",
    "            train_op = bert.optimization.create_optimizer(\n",
    "            loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)\n",
    "\n",
    "            # Calculate evaluation metrics. \n",
    "            def metric_fn(label_ids, predicted_labels):\n",
    "                accuracy = tf.metrics.accuracy(label_ids, predicted_labels)\n",
    "                f1_score = tf.contrib.metrics.f1_score(label_ids, predicted_labels)\n",
    "                auc = tf.metrics.auc(label_ids, predicted_labels)\n",
    "                recall = tf.metrics.recall(label_ids, predicted_labels)\n",
    "                precision = tf.metrics.precision(label_ids, predicted_labels) \n",
    "                true_pos = tf.metrics.true_positives(label_ids, predicted_labels)\n",
    "                true_neg = tf.metrics.true_negatives(label_ids, predicted_labels)   \n",
    "                false_pos = tf.metrics.false_positives(label_ids, predicted_labels)  \n",
    "                false_neg = tf.metrics.false_negatives(label_ids, predicted_labels)\n",
    "                return {\n",
    "                    \"eval_accuracy\": accuracy,\n",
    "                    \"f1_score\": f1_score,\n",
    "                    \"auc\": auc,\n",
    "                    \"precision\": precision,\n",
    "                    \"recall\": recall,\n",
    "                    \"true_positives\": true_pos,\n",
    "                    \"true_negatives\": true_neg,\n",
    "                    \"false_positives\": false_pos,\n",
    "                    \"false_negatives\": false_neg\n",
    "                }\n",
    "\n",
    "            eval_metrics = metric_fn(label_ids, predicted_labels)\n",
    "\n",
    "            if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "                return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
    "            else:\n",
    "                return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metrics)\n",
    "        else:\n",
    "            (predicted_labels, log_probs) = create_model(\n",
    "            bert_model_hub, is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n",
    "\n",
    "            predictions = {'probabilities': log_probs, 'labels': predicted_labels}\n",
    "            return tf.estimator.EstimatorSpec(mode, predictions=predictions)\n",
    "\n",
    "    # Return the actual model function in the closure\n",
    "    return model_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimator_builder(bert_model_hub, OUTPUT_DIR, SAVE_SUMMARY_STEPS, SAVE_CHECKPOINTS_STEPS, \n",
    "                      label_list, LEARNING_RATE, num_train_steps, num_warmup_steps, BATCH_SIZE):\n",
    "\n",
    "    # Specify outpit directory and number of checkpoint steps to save\n",
    "    run_config = tf.estimator.RunConfig(\n",
    "        model_dir=OUTPUT_DIR,\n",
    "        save_summary_steps=SAVE_SUMMARY_STEPS,\n",
    "        save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)\n",
    "\n",
    "    model_fn = model_fn_builder(\n",
    "        bert_model_hub = bert_model_hub,\n",
    "        num_labels=len(label_list),\n",
    "        learning_rate=LEARNING_RATE,\n",
    "        num_train_steps=num_train_steps,\n",
    "        num_warmup_steps=num_warmup_steps)\n",
    "\n",
    "\n",
    "    estimator = tf.estimator.Estimator(\n",
    "        model_fn=model_fn,\n",
    "        config=run_config,\n",
    "        params={\"batch_size\": BATCH_SIZE})\n",
    "#         use_tpu=False\n",
    "#         train_batch_size=BATCH_SIZE,\n",
    "#         eval_batch_size=8,\n",
    "#         predict_batch_size=8)\n",
    "    return estimator, model_fn, run_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_on_dfs(train, test, DATA_COLUMN, LABEL_COLUMN, \n",
    "               MAX_SEQ_LENGTH = 128,\n",
    "               BATCH_SIZE = 32,\n",
    "               LEARNING_RATE = 2e-5,\n",
    "               NUM_TRAIN_EPOCHS = 3.0,\n",
    "               WARMUP_PROPORTION = 0.1,\n",
    "               SAVE_SUMMARY_STEPS = 100,\n",
    "               SAVE_CHECKPOINTS_STEPS = 10000,\n",
    "               bert_model_hub = \"https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1\"):\n",
    "\n",
    "    label_list = train[LABEL_COLUMN].unique().tolist()\n",
    "    \n",
    "    tokenizer = create_tokenizer_from_hub_module(bert_model_hub)\n",
    "\n",
    "    train_features = make_features(train, label_list, MAX_SEQ_LENGTH, tokenizer, DATA_COLUMN, LABEL_COLUMN)\n",
    "    test_features = make_features(test, label_list, MAX_SEQ_LENGTH, tokenizer, DATA_COLUMN, LABEL_COLUMN)\n",
    "\n",
    "    num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)\n",
    "    num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)\n",
    "\n",
    "    estimator, model_fn, run_config = estimator_builder(\n",
    "                                  bert_model_hub, \n",
    "                                  OUTPUT_DIR, \n",
    "                                  SAVE_SUMMARY_STEPS, \n",
    "                                  SAVE_CHECKPOINTS_STEPS, \n",
    "                                  label_list, \n",
    "                                  LEARNING_RATE, \n",
    "                                  num_train_steps, \n",
    "                                  num_warmup_steps, \n",
    "                                  BATCH_SIZE)\n",
    "\n",
    "    train_input_fn = bert.run_classifier.input_fn_builder(\n",
    "        features=train_features,\n",
    "        seq_length=MAX_SEQ_LENGTH,\n",
    "        is_training=True,\n",
    "        drop_remainder=False)\n",
    "\n",
    "    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n",
    "\n",
    "    test_input_fn = run_classifier.input_fn_builder(\n",
    "        features=test_features,\n",
    "        seq_length=MAX_SEQ_LENGTH,\n",
    "        is_training=False,\n",
    "        drop_remainder=False)\n",
    "\n",
    "    result_dict = estimator.evaluate(input_fn=test_input_fn, steps=None)\n",
    "    return result_dict, estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUTPUT_DIR = 'output'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('toutiao_cat_data.txt', 'r', encoding='utf8') as f:\n",
    "    data = f.read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = []\n",
    "title = []\n",
    "keyword = []\n",
    "\n",
    "for i in range(len(data)):\n",
    "    label.append(data[i].split('_!_')[1])\n",
    "    title.append(data[i].split('_!_')[3])\n",
    "    keyword.append(data[i].split('_!_')[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>日本凭什么拿下印度高铁订单？</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>中国人，应该对种族歧视说“不”</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>西藏旅游为何不能随便买骨制纪念品？这里告诉你答案！</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>奥迪RSQ8路试谍照，或将于明年3月发布</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>李亚鹏晒与王菲十年前旧照 曾一起肩并肩做公益</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       title label\n",
       "0             日本凭什么拿下印度高铁订单？   113\n",
       "1            中国人，应该对种族歧视说“不”   113\n",
       "2  西藏旅游为何不能随便买骨制纪念品？这里告诉你答案！   112\n",
       "3       奥迪RSQ8路试谍照，或将于明年3月发布   107\n",
       "4     李亚鹏晒与王菲十年前旧照 曾一起肩并肩做公益   102"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keyword = pd.DataFrame(keyword)\n",
    "title = pd.DataFrame(title)\n",
    "label = pd.DataFrame(label)\n",
    "trainData = pd.concat( [title , keyword], axis=1 )\n",
    "trainData = pd.concat( [trainData , label], axis=1 )\n",
    "trainData.columns = ['title', 'keyword', 'label']\n",
    "trainData = shuffle(trainData)\n",
    "\n",
    "trainData = trainData[:2000]\n",
    "trainData.index = range(len(trainData)) # row index從0開始\n",
    "trainData = trainData.drop(['keyword'], axis=1)\n",
    "trainData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 對dataset label 做編碼\n",
    "le = LabelEncoder()\n",
    "trainData.label = le.fit_transform(trainData.label).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>日本凭什么拿下印度高铁订单？</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>中国人，应该对种族歧视说“不”</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>西藏旅游为何不能随便买骨制纪念品？这里告诉你答案！</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>奥迪RSQ8路试谍照，或将于明年3月发布</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>李亚鹏晒与王菲十年前旧照 曾一起肩并肩做公益</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       title  label\n",
       "0             日本凭什么拿下印度高铁订单？     11\n",
       "1            中国人，应该对种族歧视说“不”     11\n",
       "2  西藏旅游为何不能随便买骨制纪念品？这里告诉你答案！     10\n",
       "3       奥迪RSQ8路试谍照，或将于明年3月发布      6\n",
       "4     李亚鹏晒与王菲十年前旧照 曾一起肩并肩做公益      2"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 随机选择20%作为测试集，剩余作为训练集\n",
    "x_train, x_test, y_train, y_test = train_test_split(trainData.title, trainData.label, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = pd.DataFrame(x_train)\n",
    "x_test = pd.DataFrame(x_test)\n",
    "y_train = pd.DataFrame(y_train)\n",
    "y_test = pd.DataFrame(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat( [x_train , y_train], axis=1 )   # axis=1是X軸，axis=0是y軸\n",
    "test = pd.concat( [x_test , y_test], axis=1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>582</th>\n",
       "      <td>天生一对，美好大结局</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>5.9涨停预测</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1827</th>\n",
       "      <td>LOL版块迎来JAG战队 斗鱼首秀开启倒计时</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>上联：南来北往东西客，如何对下联？</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>708</th>\n",
       "      <td>中超第十轮，江苏苏宁1:0战胜大连一方挺进联赛前三，你如何评价本场比赛？</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     title  label\n",
       "582                             天生一对，美好大结局      0\n",
       "159                                5.9涨停预测      4\n",
       "1827                LOL版块迎来JAG战队 斗鱼首秀开启倒计时     14\n",
       "318                      上联：南来北往东西客，如何对下联？      1\n",
       "708   中超第十轮，江苏苏宁1:0战胜大连一方挺进联赛前三，你如何评价本场比赛？      3"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>北斗导航已经很厉害了，为什么国产手机还在用GPS导航？</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>未来自动驾驶真的会让酒驾和疲劳驾驶成历史吗？</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1132</th>\n",
       "      <td>华为任正非如何管理低绩效员工？</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>731</th>\n",
       "      <td>Google I/O大会落幕，国内一加6将首批支持Android P</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1754</th>\n",
       "      <td>你们对高考的独家记忆是什么？</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   title  label\n",
       "405          北斗导航已经很厉害了，为什么国产手机还在用GPS导航？      8\n",
       "1190              未来自动驾驶真的会让酒驾和疲劳驾驶成历史吗？      6\n",
       "1132                     华为任正非如何管理低绩效员工？      8\n",
       "731   Google I/O大会落幕，国内一加6将首批支持Android P      8\n",
       "1754                      你们对高考的独家记忆是什么？      7"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# myparam = {\n",
    "#         \"DATA_COLUMN\": \"title\",\n",
    "#         \"LABEL_COLUMN\": \"label\",\n",
    "#         \"LEARNING_RATE\": 2e-5,\n",
    "#         \"NUM_TRAIN_EPOCHS\":3,\n",
    "#         \"bert_model_hub\":\"https://tfhub.dev/google/bert_chinese_L-12_H-768_A-12/1\"\n",
    "#     }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result, estimator = run_on_dfs(train, test, **myparam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_COLUMN = \"title\"\n",
    "LABEL_COLUMN = \"label\" \n",
    "MAX_SEQ_LENGTH = 128\n",
    "BATCH_SIZE = 32\n",
    "LEARNING_RATE = 2e-5,\n",
    "NUM_TRAIN_EPOCHS = 3.0\n",
    "WARMUP_PROPORTION = 0.1\n",
    "SAVE_SUMMARY_STEPS = 100\n",
    "SAVE_CHECKPOINTS_STEPS = 10000\n",
    "bert_model_hub = \"https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_list = train[LABEL_COLUMN].unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0804 19:50:41.916016 11296 deprecation_wrapper.py:119] From D:\\Anaconda\\lib\\site-packages\\bert\\tokenization.py:125: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = create_tokenizer_from_hub_module(bert_model_hub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0804 19:50:42.038173 11296 deprecation_wrapper.py:119] From D:\\Anaconda\\lib\\site-packages\\bert\\run_classifier.py:774: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_features = make_features(train, label_list, MAX_SEQ_LENGTH, tokenizer, DATA_COLUMN, LABEL_COLUMN)\n",
    "test_features = make_features(test, label_list, MAX_SEQ_LENGTH, tokenizer, DATA_COLUMN, LABEL_COLUMN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)\n",
    "num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator, model_fn, run_config = estimator_builder(\n",
    "                                bert_model_hub, \n",
    "                                OUTPUT_DIR, \n",
    "                                SAVE_SUMMARY_STEPS, \n",
    "                                SAVE_CHECKPOINTS_STEPS, \n",
    "                                label_list, \n",
    "                                LEARNING_RATE, \n",
    "                                num_train_steps, \n",
    "                                num_warmup_steps, \n",
    "                                BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input_fn = bert.run_classifier.input_fn_builder(\n",
    "    features=train_features,\n",
    "    seq_length=MAX_SEQ_LENGTH,\n",
    "    is_training=True,\n",
    "    drop_remainder=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# early_stopping_hook = tf.estimator.experimental.stop_if_no_decrease_hook(\n",
    "#             estimator=estimator,\n",
    "#             metric_name='eval_loss',\n",
    "#             max_steps_without_decrease=num_train_steps,\n",
    "#             eval_dir=None,\n",
    "#             min_steps=0,\n",
    "#             run_every_secs=None,\n",
    "#             run_every_steps=500      #FLAGS.save_checkpoints_steps\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0804 19:50:42.759582 11296 deprecation.py:323] From D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
      "W0804 19:50:45.927265 11296 deprecation.py:506] From <ipython-input-5-845caca54516>:25: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0804 19:50:45.968246 11296 deprecation_wrapper.py:119] From D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "W0804 19:50:45.970223 11296 deprecation_wrapper.py:119] From D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "W0804 19:50:45.976232 11296 deprecation.py:323] From D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\keras\\optimizer_v2\\learning_rate_schedule.py:409: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "W0804 19:50:45.988234 11296 deprecation_wrapper.py:119] From D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n",
      "W0804 19:50:46.156116 11296 deprecation.py:323] From D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1205: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W0804 19:50:48.701188 11296 deprecation_wrapper.py:119] From D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:117: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py:93: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n",
      "W0804 19:50:52.741207 11296 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W0804 19:50:56.794858 11296 deprecation.py:323] From D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "Restoring from checkpoint failed. This is most likely due to a mismatch between the current graph and the graph from the checkpoint. Please ensure that you have not altered the graph expected based on the checkpoint. Original error:\n\nAssign requires shapes of both tensors to match. lhs shape= [30522,768] rhs shape= [21128,768]\n\t [[node save_2/Assign_10 (defined at <ipython-input-31-abb4a36e3c93>:1) ]]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node save_2/Assign_10:\n module/bert/embeddings/word_embeddings/adam_v (defined at D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:128)\n\nOriginal stack trace for 'save_2/Assign_10':\n  File \"D:\\Anaconda\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"D:\\Anaconda\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"D:\\Anaconda\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 499, in start\n    self.io_loop.start()\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 132, in start\n    self.asyncio_loop.run_forever()\n  File \"D:\\Anaconda\\lib\\asyncio\\base_events.py\", line 539, in run_forever\n    self._run_once()\n  File \"D:\\Anaconda\\lib\\asyncio\\base_events.py\", line 1775, in _run_once\n    handle._run()\n  File \"D:\\Anaconda\\lib\\asyncio\\events.py\", line 88, in _run\n    self._context.run(self._callback, *self._args)\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\ioloop.py\", line 758, in _run_callback\n    ret = callback()\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 536, in <lambda>\n    self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2662, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2785, in _run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2907, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2961, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-31-abb4a36e3c93>\", line 1, in <module>\n    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 367, in train\n    loss = self._train_model(input_fn, hooks, saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1158, in _train_model\n    return self._train_model_default(input_fn, hooks, saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1192, in _train_model_default\n    saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1480, in _train_with_estimator_spec\n    log_step_count_steps=log_step_count_steps) as mon_sess:\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 584, in MonitoredTrainingSession\n    stop_grace_period_secs=stop_grace_period_secs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1007, in __init__\n    stop_grace_period_secs=stop_grace_period_secs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 725, in __init__\n    self._sess = _RecoverableSession(self._coordinated_creator)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1200, in __init__\n    _WrappedSession.__init__(self, self._create_session())\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1205, in _create_session\n    return self._sess_creator.create_session()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 871, in create_session\n    self.tf_sess = self._session_creator.create_session()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 638, in create_session\n    self._scaffold.finalize()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 237, in finalize\n    self._saver.build()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 837, in build\n    self._build(self._filename, build_save=True, build_restore=True)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 875, in _build\n    build_restore=build_restore)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 502, in _build_internal\n    restore_sequentially, reshape)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 381, in _AddShardedRestoreOps\n    name=\"restore_shard\"))\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 350, in _AddRestoreOps\n    assign_ops.append(saveable.restore(saveable_tensors, shapes))\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saving\\saveable_object_util.py\", line 72, in restore\n    self.op.get_shape().is_fully_defined())\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\state_ops.py\", line 227, in assign\n    validate_shape=validate_shape)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\gen_state_ops.py\", line 69, in assign\n    use_locking=use_locking, name=name)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1355\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1356\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1357\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1341\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1342\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1429\u001b[1;33m         run_metadata)\n\u001b[0m\u001b[0;32m   1430\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Assign requires shapes of both tensors to match. lhs shape= [30522,768] rhs shape= [21128,768]\n\t [[{{node save_2/Assign_10}}]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, sess, save_path)\u001b[0m\n\u001b[0;32m   1285\u001b[0m         sess.run(self.saver_def.restore_op_name,\n\u001b[1;32m-> 1286\u001b[1;33m                  {self.saver_def.filename_tensor_name: save_path})\n\u001b[0m\u001b[0;32m   1287\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mNotFoundError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 950\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    951\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1173\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1174\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1350\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1351\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1369\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0merror_interpolation\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minterpolate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1370\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1371\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Assign requires shapes of both tensors to match. lhs shape= [30522,768] rhs shape= [21128,768]\n\t [[node save_2/Assign_10 (defined at <ipython-input-31-abb4a36e3c93>:1) ]]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node save_2/Assign_10:\n module/bert/embeddings/word_embeddings/adam_v (defined at D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:128)\n\nOriginal stack trace for 'save_2/Assign_10':\n  File \"D:\\Anaconda\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"D:\\Anaconda\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"D:\\Anaconda\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 499, in start\n    self.io_loop.start()\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 132, in start\n    self.asyncio_loop.run_forever()\n  File \"D:\\Anaconda\\lib\\asyncio\\base_events.py\", line 539, in run_forever\n    self._run_once()\n  File \"D:\\Anaconda\\lib\\asyncio\\base_events.py\", line 1775, in _run_once\n    handle._run()\n  File \"D:\\Anaconda\\lib\\asyncio\\events.py\", line 88, in _run\n    self._context.run(self._callback, *self._args)\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\ioloop.py\", line 758, in _run_callback\n    ret = callback()\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 536, in <lambda>\n    self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2662, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2785, in _run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2907, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2961, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-31-abb4a36e3c93>\", line 1, in <module>\n    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 367, in train\n    loss = self._train_model(input_fn, hooks, saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1158, in _train_model\n    return self._train_model_default(input_fn, hooks, saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1192, in _train_model_default\n    saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1480, in _train_with_estimator_spec\n    log_step_count_steps=log_step_count_steps) as mon_sess:\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 584, in MonitoredTrainingSession\n    stop_grace_period_secs=stop_grace_period_secs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1007, in __init__\n    stop_grace_period_secs=stop_grace_period_secs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 725, in __init__\n    self._sess = _RecoverableSession(self._coordinated_creator)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1200, in __init__\n    _WrappedSession.__init__(self, self._create_session())\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1205, in _create_session\n    return self._sess_creator.create_session()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 871, in create_session\n    self.tf_sess = self._session_creator.create_session()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 638, in create_session\n    self._scaffold.finalize()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 237, in finalize\n    self._saver.build()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 837, in build\n    self._build(self._filename, build_save=True, build_restore=True)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 875, in _build\n    build_restore=build_restore)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 502, in _build_internal\n    restore_sequentially, reshape)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 381, in _AddShardedRestoreOps\n    name=\"restore_shard\"))\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 350, in _AddRestoreOps\n    assign_ops.append(saveable.restore(saveable_tensors, shapes))\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saving\\saveable_object_util.py\", line 72, in restore\n    self.op.get_shape().is_fully_defined())\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\state_ops.py\", line 227, in assign\n    validate_shape=validate_shape)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\gen_state_ops.py\", line 69, in assign\n    use_locking=use_locking, name=name)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-abb4a36e3c93>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrain_input_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_steps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnum_train_steps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, input_fn, hooks, steps, max_steps, saving_listeners)\u001b[0m\n\u001b[0;32m    365\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    366\u001b[0m       \u001b[0msaving_listeners\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_check_listeners_type\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msaving_listeners\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 367\u001b[1;33m       \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_train_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    368\u001b[0m       \u001b[0mlogging\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Loss for final step: %s.'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    369\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\u001b[0m in \u001b[0;36m_train_model\u001b[1;34m(self, input_fn, hooks, saving_listeners)\u001b[0m\n\u001b[0;32m   1156\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_train_model_distributed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1157\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1158\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_train_model_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1159\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1160\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_train_model_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\u001b[0m in \u001b[0;36m_train_model_default\u001b[1;34m(self, input_fn, hooks, saving_listeners)\u001b[0m\n\u001b[0;32m   1190\u001b[0m       return self._train_with_estimator_spec(estimator_spec, worker_hooks,\n\u001b[0;32m   1191\u001b[0m                                              \u001b[0mhooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mglobal_step_tensor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1192\u001b[1;33m                                              saving_listeners)\n\u001b[0m\u001b[0;32m   1193\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1194\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_train_model_distributed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaving_listeners\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\u001b[0m in \u001b[0;36m_train_with_estimator_spec\u001b[1;34m(self, estimator_spec, worker_hooks, hooks, global_step_tensor, saving_listeners)\u001b[0m\n\u001b[0;32m   1478\u001b[0m         \u001b[0msave_summaries_steps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msave_summary_steps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1479\u001b[0m         \u001b[0mconfig\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session_config\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1480\u001b[1;33m         log_step_count_steps=log_step_count_steps) as mon_sess:\n\u001b[0m\u001b[0;32m   1481\u001b[0m       \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1482\u001b[0m       \u001b[0many_step_done\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\u001b[0m in \u001b[0;36mMonitoredTrainingSession\u001b[1;34m(master, is_chief, checkpoint_dir, scaffold, hooks, chief_only_hooks, save_checkpoint_secs, save_summaries_steps, save_summaries_secs, config, stop_grace_period_secs, log_step_count_steps, max_wait_secs, save_checkpoint_steps, summary_dir)\u001b[0m\n\u001b[0;32m    582\u001b[0m       \u001b[0msession_creator\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msession_creator\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    583\u001b[0m       \u001b[0mhooks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mall_hooks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 584\u001b[1;33m       stop_grace_period_secs=stop_grace_period_secs)\n\u001b[0m\u001b[0;32m    585\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    586\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, session_creator, hooks, stop_grace_period_secs)\u001b[0m\n\u001b[0;32m   1005\u001b[0m         \u001b[0mhooks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1006\u001b[0m         \u001b[0mshould_recover\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1007\u001b[1;33m         stop_grace_period_secs=stop_grace_period_secs)\n\u001b[0m\u001b[0;32m   1008\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1009\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, session_creator, hooks, should_recover, stop_grace_period_secs)\u001b[0m\n\u001b[0;32m    723\u001b[0m         stop_grace_period_secs=stop_grace_period_secs)\n\u001b[0;32m    724\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mshould_recover\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 725\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sess\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_RecoverableSession\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_coordinated_creator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    726\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    727\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sess\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_coordinated_creator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, sess_creator)\u001b[0m\n\u001b[0;32m   1198\u001b[0m     \"\"\"\n\u001b[0;32m   1199\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sess_creator\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess_creator\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1200\u001b[1;33m     \u001b[0m_WrappedSession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_create_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1201\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1202\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_create_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\u001b[0m in \u001b[0;36m_create_session\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1203\u001b[0m     \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1204\u001b[0m       \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1205\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sess_creator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1206\u001b[0m       \u001b[1;32mexcept\u001b[0m \u001b[0m_PREEMPTION_ERRORS\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1207\u001b[0m         logging.info(\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\u001b[0m in \u001b[0;36mcreate_session\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    869\u001b[0m       \u001b[1;34m\"\"\"Creates a coordinated session.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    870\u001b[0m       \u001b[1;31m# Keep the tf_sess for unit testing.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 871\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtf_sess\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session_creator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    872\u001b[0m       \u001b[1;31m# We don't want coordinator to suppress any exception.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    873\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcoord\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcoordinator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCoordinator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclean_stop_exception_types\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\u001b[0m in \u001b[0;36mcreate_session\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    645\u001b[0m         \u001b[0minit_op\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_scaffold\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minit_op\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    646\u001b[0m         \u001b[0minit_feed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_scaffold\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minit_feed_dict\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 647\u001b[1;33m         init_fn=self._scaffold.init_fn)\n\u001b[0m\u001b[0;32m    648\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    649\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\session_manager.py\u001b[0m in \u001b[0;36mprepare_session\u001b[1;34m(self, master, init_op, saver, checkpoint_dir, checkpoint_filename_with_path, wait_for_checkpoint, max_wait_secs, config, init_feed_dict, init_fn)\u001b[0m\n\u001b[0;32m    288\u001b[0m         \u001b[0mwait_for_checkpoint\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mwait_for_checkpoint\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    289\u001b[0m         \u001b[0mmax_wait_secs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_wait_secs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 290\u001b[1;33m         config=config)\n\u001b[0m\u001b[0;32m    291\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_loaded_from_checkpoint\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0minit_op\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0minit_fn\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_local_init_op\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\session_manager.py\u001b[0m in \u001b[0;36m_restore_checkpoint\u001b[1;34m(self, master, saver, checkpoint_dir, checkpoint_filename_with_path, wait_for_checkpoint, max_wait_secs, config)\u001b[0m\n\u001b[0;32m    218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    219\u001b[0m     \u001b[1;31m# Loads the checkpoint.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 220\u001b[1;33m     \u001b[0msaver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msess\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mckpt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel_checkpoint_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    221\u001b[0m     \u001b[0msaver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecover_last_checkpoints\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mckpt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mall_model_checkpoint_paths\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    222\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, sess, save_path)\u001b[0m\n\u001b[0;32m   1320\u001b[0m       \u001b[1;31m# We add a more reasonable error message here to help users (b/110263146)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1321\u001b[0m       raise _wrap_restore_error_with_msg(\n\u001b[1;32m-> 1322\u001b[1;33m           err, \"a mismatch between the current graph and the graph\")\n\u001b[0m\u001b[0;32m   1323\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1324\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Restoring from checkpoint failed. This is most likely due to a mismatch between the current graph and the graph from the checkpoint. Please ensure that you have not altered the graph expected based on the checkpoint. Original error:\n\nAssign requires shapes of both tensors to match. lhs shape= [30522,768] rhs shape= [21128,768]\n\t [[node save_2/Assign_10 (defined at <ipython-input-31-abb4a36e3c93>:1) ]]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node save_2/Assign_10:\n module/bert/embeddings/word_embeddings/adam_v (defined at D:\\Anaconda\\lib\\site-packages\\bert\\optimization.py:128)\n\nOriginal stack trace for 'save_2/Assign_10':\n  File \"D:\\Anaconda\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"D:\\Anaconda\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"D:\\Anaconda\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 499, in start\n    self.io_loop.start()\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 132, in start\n    self.asyncio_loop.run_forever()\n  File \"D:\\Anaconda\\lib\\asyncio\\base_events.py\", line 539, in run_forever\n    self._run_once()\n  File \"D:\\Anaconda\\lib\\asyncio\\base_events.py\", line 1775, in _run_once\n    handle._run()\n  File \"D:\\Anaconda\\lib\\asyncio\\events.py\", line 88, in _run\n    self._context.run(self._callback, *self._args)\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\ioloop.py\", line 758, in _run_callback\n    ret = callback()\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 536, in <lambda>\n    self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"D:\\Anaconda\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2662, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2785, in _run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2907, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"D:\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2961, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-31-abb4a36e3c93>\", line 1, in <module>\n    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 367, in train\n    loss = self._train_model(input_fn, hooks, saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1158, in _train_model\n    return self._train_model_default(input_fn, hooks, saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1192, in _train_model_default\n    saving_listeners)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow_estimator\\python\\estimator\\estimator.py\", line 1480, in _train_with_estimator_spec\n    log_step_count_steps=log_step_count_steps) as mon_sess:\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 584, in MonitoredTrainingSession\n    stop_grace_period_secs=stop_grace_period_secs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1007, in __init__\n    stop_grace_period_secs=stop_grace_period_secs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 725, in __init__\n    self._sess = _RecoverableSession(self._coordinated_creator)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1200, in __init__\n    _WrappedSession.__init__(self, self._create_session())\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 1205, in _create_session\n    return self._sess_creator.create_session()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 871, in create_session\n    self.tf_sess = self._session_creator.create_session()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 638, in create_session\n    self._scaffold.finalize()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py\", line 237, in finalize\n    self._saver.build()\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 837, in build\n    self._build(self._filename, build_save=True, build_restore=True)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 875, in _build\n    build_restore=build_restore)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 502, in _build_internal\n    restore_sequentially, reshape)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 381, in _AddShardedRestoreOps\n    name=\"restore_shard\"))\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 350, in _AddRestoreOps\n    assign_ops.append(saveable.restore(saveable_tensors, shapes))\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saving\\saveable_object_util.py\", line 72, in restore\n    self.op.get_shape().is_fully_defined())\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\state_ops.py\", line 227, in assign\n    validate_shape=validate_shape)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\gen_state_ops.py\", line 69, in assign\n    use_locking=use_locking, name=name)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"D:\\Anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n"
     ]
    }
   ],
   "source": [
    "estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_input_fn = run_classifier.input_fn_builder(\n",
    "    features=test_features,\n",
    "    seq_length=MAX_SEQ_LENGTH,\n",
    "    is_training=False,\n",
    "    drop_remainder=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_dict = estimator.evaluate(input_fn=test_input_fn, steps=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretty_print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
